{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import torch\n",
    "torch.set_printoptions(edgeitems=2, linewidth=75)\n",
    "import torch.optim as optim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "t_c = torch.tensor([0.5, 14.0, 15.0, 28.0, 11.0,\n",
    "                    8.0, 3.0, -4.0, 6.0, 13.0, 21.0])\n",
    "t_u = torch.tensor([35.7, 55.9, 58.2, 81.9, 56.3, 48.9,\n",
    "                    33.9, 21.8, 48.4, 60.4, 68.4])\n",
    "t_un = 0.1 * t_u\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def model(t_u,w,b):\n",
    "    return w * t_u + b\n",
    "\n",
    "def loss_fn(t_p,t_c):\n",
    "    squared_diffs = (t_p - t_c)**2\n",
    "    return squared_diffs.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['ASGD',\n",
       " 'Adadelta',\n",
       " 'Adagrad',\n",
       " 'Adam',\n",
       " 'AdamW',\n",
       " 'Adamax',\n",
       " 'LBFGS',\n",
       " 'Optimizer',\n",
       " 'RMSprop',\n",
       " 'Rprop',\n",
       " 'SGD',\n",
       " 'SparseAdam',\n",
       " '__builtins__',\n",
       " '__cached__',\n",
       " '__doc__',\n",
       " '__file__',\n",
       " '__loader__',\n",
       " '__name__',\n",
       " '__package__',\n",
       " '__path__',\n",
       " '__spec__',\n",
       " 'lr_scheduler']"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dir(optim)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "params = torch.tensor([1.0,0.0],requires_grad = True)\n",
    "learning_rate = 1e-5\n",
    "optimizer = optim.SGD([params],lr = learning_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([ 9.5483e-01, -8.2600e-04], requires_grad=True)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t_p = model(t_u,*params)\n",
    "loss = loss_fn(t_p,t_c)\n",
    "loss.backward()\n",
    "\n",
    "\n",
    "optimizer.step()\n",
    "params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# loop\n",
    "def training_loop(n_epochs,optimizer,params,t_u,t_c):\n",
    "    for epoch in range(1,n_epochs+1):\n",
    "        t_p = model(t_u,*params)\n",
    "        loss = loss_fn(t_p,t_c)\n",
    "        \n",
    "        \n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        \n",
    "        if epoch % 500 == 0:\n",
    "            print('Epoch %d,Loss %f' % (epoch,float(loss)))\n",
    "            \n",
    "    return params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 500,Loss 7.860118\n",
      "Epoch 1000,Loss 3.828538\n",
      "Epoch 1500,Loss 3.092191\n",
      "Epoch 2000,Loss 2.957697\n",
      "Epoch 2500,Loss 2.933134\n",
      "Epoch 3000,Loss 2.928648\n",
      "Epoch 3500,Loss 2.927830\n",
      "Epoch 4000,Loss 2.927680\n",
      "Epoch 4500,Loss 2.927651\n",
      "Epoch 5000,Loss 2.927648\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "tensor([  5.3671, -17.3012], requires_grad=True)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "params = torch.tensor([1.0,0.0],requires_grad = True)\n",
    "learning_rate = 1e-2\n",
    "optimizer = optim.SGD([params],lr = learning_rate)\n",
    "\n",
    "training_loop(5000,optimizer,params,t_un,t_c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 500,Loss 7.612903\n",
      "Epoch 1000,Loss 3.086700\n",
      "Epoch 1500,Loss 2.928578\n",
      "Epoch 2000,Loss 2.927646\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "tensor([  0.5367, -17.3021], requires_grad=True)"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 使用ADAM\n",
    "\n",
    "params = torch.tensor([1.0,0.0],requires_grad = True)\n",
    "learning_rate = 1e-1\n",
    "optimizer = optim.Adam([params],lr = learning_rate)\n",
    "\n",
    "training_loop(2000,optimizer,params,t_u,t_c)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 评估损失（训练集）\n",
    "规则：\n",
    "1. 如果训练损失没有下降，可能是因为模型对于数据来说过于简单，另一个可能就是我们的数据没有包含有效信息\n",
    "2. 如果训练损失和验证损失发散（趋势不同），证明模型过拟合了"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([ 7,  8,  4,  2, 10,  3,  6,  1,  5]), tensor([0, 9]))"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 分离数据集\n",
    "\n",
    "n_samples = t_u.shape[0]\n",
    "n_val = int(0.2*n_samples)\n",
    "\n",
    "shuffled_indices = torch.randperm(n_samples)    # 随机排列样本的索引\n",
    "\n",
    "train_indices = shuffled_indices[:-n_val]       # 取前几个作为训练集（得到其索引）\n",
    "val_indices = shuffled_indices[-n_val:]\n",
    "\n",
    "train_indices,val_indices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 取对应的数据\n",
    "\n",
    "train_t_u = t_u[train_indices]\n",
    "train_t_c = t_c[train_indices]\n",
    "\n",
    "val_t_u = t_u[val_indices]\n",
    "val_t_c = t_c[val_indices]\n",
    "\n",
    "train_t_un = 0.1*train_t_u\n",
    "val_t_un = 0.1*val_t_u"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "def training_loop(n_epochs,optimizer,params,train_t_u,val_t_u,train_t_c,val_t_c):\n",
    "    for epoch in range(1,n_epochs+1):\n",
    "        train_t_p = model(train_t_u,*params)\n",
    "        train_loss = loss_fn(train_t_p,train_t_c)\n",
    "        with torch.no_grad():                         # 验证时不产生额外参数\n",
    "            val_t_p = model(val_t_u,*params)\n",
    "            val_loss = loss_fn(val_t_p,val_t_c)\n",
    "            assert val_loss.requires_grad == False\n",
    "        \n",
    "        optimizer.zero_grad()\n",
    "        train_loss.backward()\n",
    "        optimizer.step()\n",
    "        \n",
    "        if epoch <=3 or epoch % 500 == 0:\n",
    "            print(f\"Epoch {epoch}, Training loss {train_loss.item():.4f},\"\n",
    "                  f\" Validation loss {val_loss.item():.4f}\")\n",
    "            \n",
    "    return params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1, Training loss 91.7935, Validation loss 28.9332\n",
      "Epoch 2, Training loss 82.2039, Validation loss 25.7886\n",
      "Epoch 3, Training loss 73.4814, Validation loss 23.3551\n",
      "Epoch 500, Training loss 2.7724, Validation loss 5.1029\n",
      "Epoch 1000, Training loss 2.7248, Validation loss 4.6795\n",
      "Epoch 1500, Training loss 2.7248, Validation loss 4.6792\n",
      "Epoch 2000, Training loss 2.7248, Validation loss 4.6792\n",
      "Epoch 2500, Training loss 2.7248, Validation loss 4.6792\n",
      "Epoch 3000, Training loss 2.7248, Validation loss 4.6792\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "tensor([  5.3414, -16.7803], requires_grad=True)"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "params = torch.tensor([1.0,0.0],requires_grad = True)\n",
    "learning_rate = 1e-1\n",
    "optimizer = optim.Adam([params],lr = learning_rate)\n",
    "\n",
    "training_loop(3000,optimizer,params,train_t_un,val_t_un,train_t_c,val_t_c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
